//页面采集
const http = require('http');
const https = require('https');
const cheerio = require('cheerio')
const fs = require('fs');
const parseurl = require('url')


//页面数据获取
class getdata {
    public outer: string | undefined;
    public middle: string | undefined;
    public content: string | undefined;
    public www: string | undefined;
    constructor(lock) {
        const { outer, middle = 'li', content, www } = lock;
        this.outer = outer;
        this.middle = middle;
        this.content = content;
        this.www = www
    }

    pachong(url) { //协议分支 ,和文件夹过滤
        if (url.includes('https')) {
            https.get(url, (res) => {
                this.analysis(res)
            })
        } else {
            http.get(url, (res) => {
                this.analysis(res)
            })
        }
    }

    geturl(url, min, max, warn = 1000) {
        // const {hostname} = httpUrl.parse(url);
        const { start, end } = url;
        const interval = setInterval(() => {

            if (min === max) {
                clearInterval(interval);
                console.log('网址采集完成');
            }
            min++;
            const url = `${start}${min}${end}`;
            console.log(url)
            console.log(`当前为：${min}页`);
            this.pachong(url);

        }, warn);
    }

    url(url) {
        this.pachong(url);
    }

    analysis(data) {
        const { statusCode } = data;
        const contentType = data.headers['content-type'];//文件类型
        let err = null;
        if (statusCode !== 200) {
            err = new Error(`状态码：${statusCode}     msg:请求错误`)
        } else if (!/^text\/html/.test(contentType)) {
            err = new ErrorEvent('文件类型错误')
        }

        if (err) {
            data.resume(); //重置缓存
            console.log(err)
            return false;
        }

        let htmltp = "";
        data.on('data', (chunk) => { //分段获取数据
            // console.log(chunk.toString('utf8'))
            htmltp += chunk.toString('utf8');
        })
        data.on('end', () => {
           
            const $1 = cheerio.load(htmltp);  //获取全面html
            const $ = cheerio.load($1(this.outer).html());  //获取指定片段 outer 为传入的id 或者class
            $(this.middle).each((index, tiem) => {  //中间层代码 循环
                
                const cm = $(tiem).html();
                const DOM = cheerio.load(cm);
              
                const hostads = DOM(this.content).attr('href');
                console.log(hostads)
                if (DOM(this.content).attr('href') && hostads.includes('.html')) {
                    // const sq = new SqlCont();
                    if (hostads.includes('http') || hostads.includes('https')) {
                        // sq.deWeight(hostads)
                        console.log(hostads)
                        fs.appendFile('./dataCont/fetishs.ml.txt',""+hostads+"\n",'utf8',function(err){  
                            if(err)  {  
                                console.log(err);  
                            }  
                        });
                    } else {
                        if (hostads.startsWith('/')) {
                            const st = hostads.substr(1, hostads.length) //删除字符串第一个字符
                            //    sq.deWeight(this.www+st)
                            console.log(this.www + st)
                            fs.appendFile('./dataCont/fetishs.ml.txt',""+this.www + st+"\n",'utf8',function(err){  
                                if(err)  {  
                                    console.log(err);  
                                }  
                            });
                        } else {
                            // sq.deWeight(this.www+hostads)
                            console.log(this.www + hostads)
                            fs.appendFile('./dataCont/fetishs.ml.txt',""+this.www + hostads+"\n",'utf8',function(err){  
                                if(err)  {  
                                    console.log(err);  
                                }  
                            });
                        }
                    }
                }else{
                    fs.appendFile('./dataCont/meitu.txt',""+hostads+"\n",'utf8',function(err){  
                        if(err)  {  
                            console.log(err);  
                        }  
                    });
                }
            })
        })
    }

}

function gethtml(filtration: any, protocol: string, hostname: string): any {
    const { outer, middle = 'li', content } = filtration;
    const gets = new getdata({
        outer,        //外部包裹层
        middle,        //内容中间层
        content,     //内容层
        www: `${protocol}//${hostname}/`,//当前域名
    });

    return gets;
}

/**
 * 
 * @param filtration  ：{  outer,        //外部包裹层
                            middle,        //内容中间层
                            content,     //内容层
                        }
 * @param url 
 * @param min 开始页
 * @param max 结尾页
 * @param m 间隔好久执行
 */
function GetTheLinks(filtration: any, url: string | any, min: number, max: number, m: number) {

    const { protocol, hostname } = parseurl.parse(url.start);

    const gets = gethtml(filtration, protocol, hostname);

    gets.geturl(url, min, max, m);
}

/**
 * 
 * @param filtration ：{  outer,        //外部包裹层
        middle,        //内容中间层
        content,     //内容层
    }
 * @param url 当前要获取的数据路径，也可为多个
 */
function GetTheLink(filtration: any, url: string | any) {

    //传入当前的地址用来解析他的 域名和端口
    const { protocol, hostname } = parseurl.parse(typeof url === 'string' ? url : url[1]);

    const gets = gethtml(filtration, protocol, hostname);

    if (typeof url === 'string') {
        gets.url(url);
    } else {
        // url.forEach((e: any) => {
        //     gets.url(e);
        // })
        let min=0,max=url.length;

        const interval = setInterval(() => {
            min++;
            if (min === max) {
                clearInterval(interval);
                console.log('网址采集完成');
            }
            gets.url(url[min]);
            // console.log(url[min])
            console.log(`当前为：${min}页`);

        }, 2000);

    }
}


export { GetTheLink, GetTheLinks }